# from sklearn.tree import DecisionTreeClassifier
from sklearn import tree
import math
import pandas as pd
import numpy as np
from sklearn import tree
import matplotlib as plt
data=pd.read_csv('UCI_Credit_Card.csv',index_col=[0])
#pd.set_option('display.max_rows', None)
pd.set_option('display.max_columns',None)
column={'ID':'User ID',
        'SeriousDlqin2yrs':'Good and Bad Customers',
        'RevolvingUtilizationOfUnsecuredLines':'Ratio of available credits',
        'age':'Age',
        'NumberOfTimes':'Number Of Times',
        'NumberOfTime30-59DaysPastDueNotWorse':'Number Of Time 30-59 Days Past Due ',
        'DebtRatio':'DebtRatio',
        'MonthlyIncome':'Monthly Income',
        'NumberOfOpenCreditLinesAndLoans':'Number Of Credits',
        'NumberOfTimes90DaysLate':'Number of 90 days past due loans',
        'NumberRealEstate':'Number of loans',
        'NumberRealEstateLoansOrLines':'Fixed Asset Loan Volume',
        'NumberOfTime60-89DaysPastDueNotWorse':'Number Of Time 60-89 Days Past Due',
        'NumberOfDependents':'Number Of family members',
        'DefaultRate':'Default rate',
        'GoodDebt':'Number of credits never defaulted',
        'RemainingIncome':'Monthly Remaining Income',
        'EstateLoan':'Fixed Asset Loan Ratio',
        'Comsumption Loan': 'Level of debt per capita',
        'AverageIncome':'Average income per household member',
        'AverageDebtLevel': 'Number of debt per household member',
        'CreaditUsedRatio':'Personal Credit Used Ratio',
        '30-59DaysRatio': 'Number of 30-59 day past due loans as a proportion of total past due',
        '60-89DaysRatio':  'Number of 60-89 day past due loans as a proportion of total past due',
        'Debt1': 'Whether over-indebted',
        'CreaditRatio':'Whether credit limit is over utilized'}
# data.rename(columns=column,inplace=True)
rs1 = 16

print(data.describe())
# print(data.info())

data.drop_duplicates(inplace=True)


# Outlier handling

LIMIT_BAL_=data['LIMIT_BAL'].mean()+3*data['LIMIT_BAL'].std()
data = data[data['LIMIT_BAL'] <= LIMIT_BAL_]

BILL_AMT=['BILL_AMT1','BILL_AMT2','BILL_AMT3','BILL_AMT4','BILL_AMT5','BILL_AMT6']
PAY_AMT=['PAY_AMT1','PAY_AMT2','PAY_AMT3','PAY_AMT4','PAY_AMT5','PAY_AMT6']
for i in BILL_AMT:
    BILL_AMT_=data[i].mean()+3*data[i].std()
    data = data[data[i] <= BILL_AMT_]
for i in PAY_AMT:
    PAY_AMT_=data[i].mean()+3*data[i].std()
    data = data[data[i] <= PAY_AMT_]



data = data.drop(['SEX'], axis=1)
data = data.drop(['MARRIAGE'], axis=1)
# decision tree bining
def optimal_binning_boundary(x: pd.Series, y: pd.Series, nan: float = -999.) -> list:

    # Obtaining a list of boundary values for the optimal bins using a decision tree
 
    boundary = []
    x = x.values  # complete missing value
    y = y.values

    clf = tree.DecisionTreeClassifier(criterion='entropy',  # smallest “entropy” criterion
                                 max_depth = 100,
                                 # max_leaf_nodes=20 ,
                                 max_leaf_nodes=22 ,    # opitimal
                                 min_samples_leaf=0.005,
                                 # random_state=rs2
                                      )

    clf.fit(x.reshape(-1, 1), y)
    # print(tree.plot_tree(clf,filled=True))

    # plt.show()

    n_nodes = clf.tree_.node_count
    children_left = clf.tree_.children_left
    children_right = clf.tree_.children_right
    threshold = clf.tree_.threshold

    for i in range(n_nodes):
        if children_left[i] != children_right[i]:  # boundary values for the optimal bins
            boundary.append(threshold[i])
    print(boundary)
    boundary.sort()

    min_x = x.min()
    max_x = x.max() + 0.1
    boundary = [min_x] + boundary + [max_x]
    print(boundary)
    return boundary
# calculate IV
def feature_woe_iv(x: pd.Series, y: pd.Series, column, data) -> pd.DataFrame:


    # x = x.fillna(nan)
    boundary = optimal_binning_boundary(x, y)  # boundary values for the optimal bins
    print(boundary)
    df = pd.concat([x, y], axis=1)
    df.columns = ['x', 'y']
    data.loc[:, column] = pd.cut(data.loc[:, column], bins=boundary, labels=[i for i in range(len(boundary) - 1)],include_lowest=True)
    df['bins'] = pd.cut(x=x, bins=boundary, right=False)

    grouped = df.groupby('bins')['y']  # Counting the number of clients
    result_df = grouped.agg([('good', lambda y: (y == 0).sum()),
                             ('bad', lambda y: (y == 1).sum()),
                             ('total', 'count')])

    result_df['good_pct'] = result_df['good'] / result_df['good'].sum()  # Percentage of customers not in default
    result_df['bad_pct'] = result_df['bad'] / result_df['bad'].sum()  # Percentage of customers in default
    result_df['total_pct'] = result_df['total'] / result_df['total'].sum()

    result_df['bad_rate'] = result_df['bad'] / result_df['total']  # default rate

    result_df['woe'] = np.log(result_df['good_pct'] / result_df['bad_pct'])  # WOE
    result_df['iv'] = (result_df['good_pct'] - result_df['bad_pct']) * result_df['woe']  # IV
    if result_df['iv'].sum() <= -0.05:
        data = data.drop([column], axis=1)
    print(column + f" IV = {result_df['iv'].sum()}")

    return data

data.insert(0, 'default.payment.next.month', data.pop('default.payment.next.month'))
# print(data.describe())
# print(data.info())


for column in data.iloc[:,1:].columns: # calculate IV
    data = feature_woe_iv(x=data[column], y=data['default.payment.next.month'],column =column ,data=data)

data = data.reset_index(drop=True)
# data.to_csv('cs-training1.csv', index=1)


# Dataset segmentation

from sklearn.model_selection import StratifiedShuffleSplit
split = StratifiedShuffleSplit(n_splits=10, test_size=0.3, random_state=rs1) # 0.3
# Sampling according to mnist["target"]
for train_index, test_index in split.split(data.iloc[:, 1:], data.iloc[:, 0]): # split_split(X,y)
    user_train = data.iloc[train_index]
    user_train_target = user_train['default.payment.next.month']
    user_test = data.iloc[test_index]
    user_test_target = user_test['default.payment.next.month']
user_train.to_csv('UCI_Credit_Card_train_DT.csv', index=1)
user_test.to_csv('UCI_Credit_Card_test_DT.csv', index=1)

print(user_train.info())
